In [1]:
drive_path = 'c:/'
import numpy as np
import pandas as pd
import os
import sys
import matplotlib.pyplot as plt
%matplotlib inline
from scipy.stats import ks_2samp
from scipy.stats import anderson_ksamp
from scipy.stats import kruskal
from scipy.stats import variation
from scipy import signal as sps
import seaborn as sns
import glob
import re
In [3]:
# (pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160330_3\\44dtnew.txt'))
# badfile=(pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160330_3\\44dt.txt'))
# rightfile=badfile.iloc[:,1:16]
# rightfile
# rightfile.to_csv('44dtnew.txt')
In [2]:
date='160525'
os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\HabituationFiles\\%s'%date)
In [3]:
#This piece spits out all the peaks from one session in one dataframe
peakdf=pd.DataFrame([])
# date='160626_2'
# os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\%s'%date)
for filename in glob.glob('*dt.txt'):
f=pd.read_csv(filename,nrows=175)
df=f[[col for col in f.columns if 'G PMT' in col]]
peak=[]
for col in df.columns:
a=df[col]
firsta=1;
firstb=24;
#Figures out if there is a min or max and sees if it passes threshold (3SD)
if np.absolute(min(a[26:80]))>np.absolute(max(a[26:80])) and np.absolute(min(a[26:80]))>=3*np.std(df[col][firsta:firstb]):
b=min(a[26:80])
peak.append(b)
elif np.absolute(max(a[26:80]))>np.absolute(min(a[26:80]))and np.absolute(max(a[26:80]))>=3*np.std(df[col][firsta:firstb]):
b=max(a[26:80])
peak.append(b)
else:
b=0
peak.append(b)
peaks=pd.DataFrame(peak).T
peaks.columns=df.columns
peaks=pd.concat([pd.DataFrame({'Trial':[int(filename.split('dt')[0])]}),peaks],axis=1)
peakdf=peakdf.append(peaks,ignore_index=True)
# peakdf.to_csv('%s_peaks.csv'%date,index=False)
In [4]:
peakdf
Out[4]:
In [8]:
trials=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Analysis\\Odor_Panel\\Odor_Trials.csv')
filerow=trials.loc[trials['File']==date]
odortrials={}
for t in filerow.Odor.unique():
y={t:[int(x) for x in filerow.loc[filerow['Odor'] == t][['T1','T2','T3','T4']].values.tolist()[0]]}
odortrials.update(y)
In [9]:
odortrials
Out[9]:
In [ ]:
In [ ]:
In [10]:
#Get average peak across all trials using peakdf dataframe
meandf=pd.DataFrame([])
for key in odortrials:
odor=odortrials[key]
mean=[]
for col in peakdf.loc[peakdf['Trial'].isin(odor)][[col for col in peakdf.loc[peakdf['Trial'].isin(odor)].columns if 'G PMT' in col]]:
mean.append(peakdf.loc[peakdf['Trial'].isin(odor)][col].mean())
mean=pd.DataFrame(mean).T
mean.columns=peakdf.loc[peakdf['Trial'].isin(odor)][[col for col in peakdf.loc[peakdf['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
meandf=meandf.append(mean)
meandf=meandf.reset_index(drop=True)
meandf.columns=[str(col)+'_'+date for col in meandf.columns]
meandf=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),meandf],axis=1)
# meandf.to_csv('%s_mean.csv'%date,index=False)
In [11]:
meandf
Out[11]:
In [7]:
#Get proportion of successful trials
successdf=pd.DataFrame([])
for key in odortrials:
odor=odortrials[key]
newdf=peakdf.loc[peakdf['Trial'].isin(odor)]
s=[]
for col in peakdf.loc[peakdf['Trial'].isin(odor)][[col for col in peakdf.loc[peakdf['Trial'].isin(odor)].columns if 'G PMT' in col]]:
s.append(np.divide((newdf.loc[:,col]!=0).sum(),float(len(newdf.loc[:,col]))))
s=pd.DataFrame(s).T
s.columns=peakdf.loc[peakdf['Trial'].isin(odor)][[col for col in peakdf.loc[peakdf['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
successdf=successdf.append(s)
successdf=successdf.reset_index(drop=True)
successdf.columns=[str(col)+'_'+date for col in successdf.columns]
successdf=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),successdf],axis=1)
successdf.to_csv('%s_success.csv'%date,index=False)
In [8]:
# # date='160330_3'
# # os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\%s'%date)
# #example dataframe
# for filename in
# df=pd.read_csv('1dt.txt')
# df=df[[col for col in df.columns if 'G PMT' in col]]
In [9]:
temp=pd.DataFrame([])
date='160330_3'
os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\%s'%date)
#Pull the trials that correspond to specific date/odors
trials=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Analysis\\Odor_Panel\\Odor_Trials.csv')
filerow=trials.loc[trials['File']==date]
odortrials={}
for t in trials.Odor.unique():
y={t:[int(x) for x in filerow.loc[filerow['Odor'] == t][['T1','T2','T3','T4']].values.tolist()[0]]}
odortrials.update(y)
#Get the frame rate for a specified date
num=trials.File.unique().tolist().index('%s'%date)
fr=trials.loc[trials['File']==trials.File.unique().tolist()[num]]['FrameRate'].iloc[0]
#Get the integral
intdf=pd.DataFrame([])
for filename in glob.glob('*dt.txt'):
f=pd.read_csv(filename,nrows=125)
df=f[[col for col in f.columns if 'G PMT' in col]]
winstart=np.int(4*fr)
winend=np.int(12*fr)
integral=[]
for col in df.columns:
a=df[col]
firsta=1;
firstb=24;
#Figures out if there is a min or max and sees if it passes threshold (3SD)
if np.absolute(min(a[26:80]))>np.absolute(max(a[26:80])) and np.absolute(min(a[26:80]))>=3*np.std(df[col][firsta:firstb]):
b=sum(df[col][winstart:winend]*(1/fr))
integral.append(b)
elif np.absolute(max(a[26:80]))>np.absolute(min(a[26:80]))and np.absolute(max(a[26:80]))>=3*np.std(df[col][firsta:firstb]):
b=sum(df[col][winstart:winend]*(1/fr))
integral.append(b)
else:
b=0
integral.append(b)
integral=pd.DataFrame(integral).T
integral.columns=df.columns
integral=pd.concat([pd.DataFrame({'Trial':[int(filename.split('dt')[0])]}),integral],axis=1)
intdf=intdf.append(integral)
# intdf.to_csv('%s_integral.csv'%date,index=False)
In [10]:
#Get average integral across all trials using integral dataframe
meanint=pd.DataFrame([])
for key in odortrials:
odor=odortrials[key]
mean=[]
for col in intdf.loc[intdf['Trial'].isin(odor)][[col for col in intdf.loc[intdf['Trial'].isin(odor)].columns if 'G PMT' in col]]:
mean.append(intdf.loc[intdf['Trial'].isin(odor)][col].mean())
mean=pd.DataFrame(mean).T
mean.columns=intdf.loc[intdf['Trial'].isin(odor)][[col for col in intdf.loc[intdf['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
meanint=meanint.append(mean)
meanint=meanint.reset_index(drop=True)
meanint.columns=[str(col)+'_'+date for col in meanint.columns]
meanint=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),meanint],axis=1)
# meanint.to_csv('%s_meanint.csv'%date,index=False)
In [ ]:
In [11]:
temp=pd.DataFrame([])
date='160330_3'
os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\%s'%date)
#Pull the trials that correspond to specific date/odors
trials=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Analysis\\Odor_Panel\\Odor_Trials.csv')
filerow=trials.loc[trials['File']==date]
odortrials={}
for t in trials.Odor.unique():
y={t:[int(x) for x in filerow.loc[filerow['Odor'] == t][['T1','T2','T3','T4']].values.tolist()[0]]}
odortrials.update(y)
#Get the frame rate for a specified date
num=trials.File.unique().tolist().index('%s'%date)
fr=trials.loc[trials['File']==trials.File.unique().tolist()[num]]['FrameRate'].iloc[0]
#Get baseline
baseline=pd.DataFrame([])
for filename in glob.glob('*dt.txt'):
f=pd.read_csv(filename,nrows=125)
df=f[[col for col in f.columns if 'G PMT' in col]]
winstart=np.int(4*fr)
winend=np.int(12*fr)
base=[]
for col in df.columns:
a=df[col]
firsta=1;
firstb=24;
b=(df[col][firsta:firstb]).mean()
base.append(b)
base=pd.DataFrame(base).T
base.columns=df.columns
base=pd.concat([pd.DataFrame({'Trial':[int(filename.split('dt')[0])]}),base],axis=1)
baseline=baseline.append(base)
# baseline.to_csv('%s_baseline.csv'%date,index=False)
In [36]:
baseline
Out[36]:
In [12]:
#mean baseline
meanbase=pd.DataFrame([])
for key in odortrials:
odor=odortrials[key]
mean=[]
for col in baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]]:
mean.append(baseline.loc[baseline['Trial'].isin(odor)][col].mean())
mean=pd.DataFrame(mean).T
mean.columns=baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
meanbase=meanbase.append(mean)
meanbase=meanbase.reset_index(drop=True)
meanbase.columns=[str(col)+'_'+date for col in meanbase.columns]
meanbase=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),meanbase],axis=1)
# meanbase.to_csv('%s_meanbase.csv'%date,index=False)
In [38]:
meanbase
Out[38]:
In [4]:
# test=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160321_1\\160321_1_baseline.csv')
In [5]:
# plt.plot(test.iloc[:,1:-1]);
In [6]:
# test2=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160321_1\\160321_1_meanbase.csv')
In [7]:
# plt.plot(test2.iloc[:,1:]);
In [8]:
# test2
In [5]:
temp=pd.DataFrame([])
date='160626_2'
os.chdir('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Untrended_For_B\\%s'%date)
#Pull the trials that correspond to specific date/odors
trials=pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Analysis\\Odor_Panel\\Odor_Trials.csv')
filerow=trials.loc[trials['File']==date]
odortrials={}
for t in trials.Odor.unique():
y={t:[int(x) for x in filerow.loc[filerow['Odor'] == t][['T1','T2','T3','T4']].values.tolist()[0]]}
odortrials.update(y)
#Get the frame rate for a specified date
num=trials.File.unique().tolist().index('%s'%date)
fr=trials.loc[trials['File']==trials.File.unique().tolist()[num]]['FrameRate'].iloc[0]
#Get baseline
baseline=pd.DataFrame([])
for filename in glob.glob('*.txt'):
f=pd.read_table(filename,skiprows=4)
df=f[[col for col in f.columns if 'G PMT (' in col]]
base=[]
for col in df.columns:
a=df[col]
firsta=2;
firstb=24;
b=(df[col][firsta:firstb]).mean()
base.append(b)
base=pd.DataFrame(base).T
base.columns=df.columns
base=pd.concat([pd.DataFrame({'Trial':[int(filename.split('.txt')[0])]}),base],axis=1)
baseline=baseline.append(base)
baseline.to_csv('%s_untrendedbaseline.csv'%date,index=False)
#mean baseline
meanbase=pd.DataFrame([])
for key in odortrials:
odor=odortrials[key]
mean=[]
for col in baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]]:
mean.append(baseline.loc[baseline['Trial'].isin(odor)][col].mean())
mean=pd.DataFrame(mean).T
mean.columns=baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
meanbase=meanbase.append(mean)
meanbase=meanbase.reset_index(drop=True)
meanbase.columns=[str(col)+'_'+date for col in meanbase.columns]
meanbase=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),meanbase],axis=1)
meanbase.to_csv('%s_meanbase.csv'%date,index=False)
In [6]:
#mean baseline
meanbase=pd.DataFrame([])
for key in odortrials:
odor=odortrials[key]
mean=[]
for col in baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]]:
mean.append(baseline.loc[baseline['Trial'].isin(odor)][col].mean())
mean=pd.DataFrame(mean).T
mean.columns=baseline.loc[baseline['Trial'].isin(odor)][[col for col in baseline.loc[baseline['Trial'].isin(odor)].columns if 'G PMT' in col]].columns
meanbase=meanbase.append(mean)
meanbase=meanbase.reset_index(drop=True)
meanbase.columns=[str(col)+'_'+date for col in meanbase.columns]
meanbase=pd.concat([pd.DataFrame({'Odor':odortrials.keys()}),meanbase],axis=1)
# meanbase.to_csv('%s_meanbase.csv'%date,index=False)
In [8]:
meanbase
Out[8]:
In [69]:
pd.read_csv('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\GoodFiles\\160325_1\\58dt.txt').size
# badfile=pd.read_table('C:\\Users\\Annie\\Documents\\Data\\Ca_Imaging\\Untrended_For_B\\160401_1\\50.txt',skiprows=4)
# rightfile=badfile.iloc[:,3:18]
# rightfile
# rightfile.to_csv('44new.csv',index=False)
In [37]:
meanbase
In [ ]: